Option Strict On

Module Spider

   Dim Log As New Logger
   Dim SystemConfig As MyDictionary

   Function Main(ByVal CmdArgs() As String) As Integer

      '''''''''''''''''''''''''''''''''''''''''
      ' System Startup
      '''''''''''''''''''''''''''''''''''''''''
      Console.WriteLine("System Starting...")

      '''''''''''''''''''''''''''''''''''''''''
      ' Input Validation
      '''''''''''''''''''''''''''''''''''''''''

      Dim Parameters As New CommandSwitches(CmdArgs)
      If Parameters("project") = "" Then
         Console.WriteLine("Please specify a project name." & vbNewLine & "Usage: spider.net.exe /project:projectname [/mode:recrawl|refresh|resume] [/log:debug|info|warn|error]")
         Terminate(1)
      End If

      '''''''''''''''''''''''''''''''''''''''''
      ' System Configuration
      '''''''''''''''''''''''''''''''''''''''''
      Console.WriteLine("Loading System Configuration...")
      Dim SystemConfig As New MyDictionary
      Try
         SystemConfig.LoadINI("config.ini")
      Catch e As Exception
         Console.WriteLine("Unable to load config.ini" & e.StackTrace)
         Terminate(1)
      End Try
      Console.WriteLine("Configuration loaded")

      '''''''''''''''''''''''''''''''''''''''''
      ' Initialise Logging
      '''''''''''''''''''''''''''''''''''''''''

      Log = New Logger(Environment.CurrentDirectory & "\" & SystemConfig("LogFolder") & Parameters("project") & ".log", Environment.CurrentDirectory & "\" & SystemConfig("LogFolder") & Parameters("project") & ".error.log", Parameters.GetSwitch("log", "warn"))
      Log.WriteToConsole = True
      If Not Log.Verify() Then
         Console.WriteLine("ERROR: Logger unable to access log files")
         Terminate(1)
      End If
      Log.Clear()

      Log.dpa("Version " & SystemConfig("version"))
      Log.dpa("Initialising Crawler...")
      Log.dpd("Initialisation Starting...")
      Log.dpd("System Configuration:" & vbNewLine & SystemConfig.ToString())

      ' Create the crawler
      Dim TheCrawler As Crawler
      Try
         Dim SpiderCrawlMax As Integer = ParseInt(Parameters.GetSwitch("max", "0"))
         TheCrawler = New Crawler(Parameters.GetSwitch("mode", "recrawl"), Log, SystemConfig, Parameters("project"), SpiderCrawlMax)
         TheCrawler.Init()
      Catch e As Exception
         Log.dpe("ERROR: Unable to initialize Crawler")
         Log.dpe(e.StackTrace)
         Terminate(1)
      End Try

      ''''''''''''''''''''''''''''''''
      ' Crawl
      ''''''''''''''''''''''''''''''''
      Log.dpa("Crawling...")
      Try
         TheCrawler.Crawl()
      Catch e As Exception
         Log.dpe("ERROR: Error during Crawl")
         Log.dpe(e.StackTrace)
         Terminate(1)
      End Try
      Log.dpa("Crawler completed")
      Log.dpa("URLs Crawled: " & TheCrawler.Total)

      ''''''''''''''''''''''''''''''''
      ' PDF2TEXT
      ''''''''''''''''''''''''''''''''
      Log.dpa("Checking for PDF2TEXT...")
      If SystemConfig("PDF2Text") <> "" Then
         Log.dpa("Running " & SystemConfig("PDF2Text"))
         Try
            Dim P2T As New PDF2TEXT(SystemConfig("connection"), Parameters("project"), SystemConfig("PDF2Text"), SystemConfig("PDF2TextParameters"))
            P2T.UpdateTextContent()
         Catch e As Exception
            Log.dpe("ERROR: During PDF2TEXT conversion")
            Log.dpe(e.StackTrace)
            Terminate(1)
         End Try
      End If

      Log.dpa("System Completed...")
#If (Debug) Then
      Console.WriteLine("Press any key...")
      Console.ReadLine()
#End If

   End Function

   '''''''''''''''''''''''''''''''''''''
   ' Program Functions
   '''''''''''''''''''''''''''''''''''''



   Sub Terminate(ByVal ErrorCode As Integer)
      ' Terminate in error

      Console.WriteLine("Abnormal termination")
#If (Debug) Then
      Console.WriteLine("Press any key...")
      Console.ReadLine()
#End If
      Environment.Exit(ErrorCode)

   End Sub



End Module
